{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.7948704, -0.6067793,  0.4894053], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pendulum-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        done = terminated or truncated\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAkz0lEQVR4nO3dfXRU9YH/8c9MHiYkYSYPkIQ0idBKxQj4wONst2pLlqipj7irlrqsUltp8Ih0PStdxVNPz4ZDf7+1tavYbc+K26rx0IpdKbTNCRi0xoCRaARNqUUTgUmAmJkEyORhvr8//DHbEdQk3GS+Sd6vc+Ycc+93vvnObZo3d+bOxGWMMQIAwELueC8AAIBPQqQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANaKW6QeffRRTZ06VSkpKVqwYIF27doVr6UAACwVl0g9++yzWr16tR588EG9/vrruvDCC1VaWqq2trZ4LAcAYClXPD5gdsGCBZo3b57+4z/+Q5IUiURUWFiou+66S/fdd99ILwcAYKnEkf6GPT09qq+v15o1a6Lb3G63SkpKVFtbe8b7hMNhhcPh6NeRSETt7e3Kzs6Wy+Ua9jUDAJxljFFnZ6fy8/Pldn/yk3ojHqmjR4+qv79fubm5Mdtzc3P1zjvvnPE+FRUV+v73vz8SywMAjKCWlhYVFBR84v4Rj9RQrFmzRqtXr45+HQwGVVRUpJaWFnm93jiuDAAwFKFQSIWFhZo4ceKnjhvxSE2aNEkJCQlqbW2N2d7a2qq8vLwz3sfj8cjj8Zy23ev1EikAGMU+6yWbEb+6Lzk5WXPmzFF1dXV0WyQSUXV1tfx+/0gvBwBgsbg83bd69WotW7ZMc+fO1fz58/WjH/1Ix48f12233RaP5QAALBWXSN100006cuSI1q5dq0AgoIsuuki/+93vTruYAgAwvsXlfVJnKxQKyefzKRgM8poUAIxCA/09zmf3AQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALDWoCO1c+dOXX311crPz5fL5dLzzz8fs98Yo7Vr12rKlCmaMGGCSkpKtH///pgx7e3tWrp0qbxerzIyMrR8+XJ1dXWd1QMBAIw9g47U8ePHdeGFF+rRRx894/7169frkUce0eOPP666ujqlpaWptLRU3d3d0TFLly7V3r17VVVVpS1btmjnzp361re+NfRHAQAYm8xZkGQ2b94c/ToSiZi8vDzzwx/+MLqto6PDeDwe88wzzxhjjNm3b5+RZHbv3h0ds23bNuNyuczBgwcH9H2DwaCRZILB4NksHwAQJwP9Pe7oa1IHDhxQIBBQSUlJdJvP59OCBQtUW1srSaqtrVVGRobmzp0bHVNSUiK32626urozzhsOhxUKhWJuAICxz9FIBQIBSVJubm7M9tzc3Oi+QCCgnJycmP2JiYnKysqKjvm4iooK+Xy+6K2wsNDJZQMALDUqru5bs2aNgsFg9NbS0hLvJQEARoCjkcrLy5Mktba2xmxvbW2N7svLy1NbW1vM/r6+PrW3t0fHfJzH45HX6425AQDGPkcjNW3aNOXl5am6ujq6LRQKqa6uTn6/X5Lk9/vV0dGh+vr66Jjt27crEolowYIFTi4HADDKJQ72Dl1dXfrzn/8c/frAgQNqaGhQVlaWioqKtGrVKv3gBz/Q9OnTNW3aND3wwAPKz8/XddddJ0k6//zzdcUVV+iOO+7Q448/rt7eXq1cuVI333yz8vPzHXtgAIAxYLCXDe7YscNIOu22bNkyY8xHl6E/8MADJjc313g8HrNo0SLT1NQUM8exY8fMLbfcYtLT043X6zW33Xab6ezsdPzSRQCAnQb6e9xljDFxbOSQhEIh+Xw+BYNBXp8CgFFooL/HR8XVfQCA8YlIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANZKjPcCAIw9vaGQTvzpTwoHAuo/cUKupCQlZ2drwrRpSikokMvlivcSMUoQKQCOiYTD+rC2Vkf/8Ad1Hzyo/q4umd5eKSFBCRMmKCkzUxkLF2ryVVcpKSuLWOEzESkAjujt6NChp57S0aoqKRKJ3dnfr/6uLvV3dSnwwQcKNTSo4PbblV5cTKjwqXhNCsBZO3HggN5/9FEdq64+PVAfZ4xO7N+vlv/8T514992RWSBGLSIF4Kz0HDmid//t3xSsq5Pp6xvw/U4eOKBDTz2lnvb2YVwdRjsiBWDI+k+e1P4HH1RPa+uQ7h+qr9exqiqZ/n6HV4axgkgBGBJjjNp37lTP0aNnNU/7yy+r59gxh1aFsYZIARg0Y4yCdXU6uHGjIt3dZzVX9/vvq//ECYdWhrGGq/sADFo4ENC769Z99kUSwFniTArAoJi+PgV+/WsChRHBmRSAAYv09OhwZaXat2+P91IwThApAANijNGHtbUK/OpXjs7rKShQwoQJjs6JsYOn+wAMSH9np1o3b3Z83qwvf1nJkyY5Pi/GBiIF4DP1hUL6y/r1OvmXvzg6b+q552pSSYlcCQmOzouxg0gB+FSRcFiHKivV+eabzk7sdss3b56SJ092dl6MKUQKwCcyxuj4/v3q+OMfHZ/bN3eucq6+2vF5MbZw4QSATxQ+eFDvVlSov7PT0XmTMjM15eablZCW5ui8GHuIFIAz6u3o0HuPPOJ4oBJ9PhWtWKG0c891dF6MTTzdB+A0kd5eHdu+XScPHHB2YpdLOVdfLe8llzg7L8YszqQAxDDGqKO2Vgd/8QvJ4U8nnzBtmnKuuUbu5GRH58XYxZkUgBjdBw/q/ccecz5Qn/+8Pv/P/6yElBRH58XYRqQARPWFQjr8zDOKnDzp6LyJPp/yb7lFnilTHJ0XYx9P9wGQ9NH7oQ4/+6w+fOklx+fOWLhQvvnz5XK5HJ8bYxtnUgA+eh1q9261vfCC43Nn+P0qvOMOAoUhIVIAFD54UId++UvH50394heV//Wvy5WU5PjcGB+IFDDOhdva9P6GDQofOuTovO4JEzR58WKlFBVxFoUh4zUpYBzrO35c7z/yiLoaGx2fe8o//IMmLV7s+LwYXziTAsYpY4xCDQ3Of3CspKzLL9fkK690fF6MP0QKGKeONzWp5fHHHZ/XM2WKJl91ldz8IUM4gKf7gHHGGKMTf/6z3vvRj9QXDDo6d0Jamgq//W2lz5jh6LwYvziTAsYZ09ur9hdfdPxCCUnK/8Y35L3oIsfnxfhFpIBx5tiOHcPyfqj0Cy5Q1qWXyuXm1wqcw9N9wDhhjFH7jh06+MQTjs+dOn26isrLlThxouNzY3zjnzzAOGCMUU9bm45t367+EyccndudmqrCb35TKZ/7nKPzAhKRAsYF09enQ5WVw3K5efZXvqK0GTN4wy6GBZECxjhjjD544gm1b9/u+NzZixbpc7feSqAwbAYVqYqKCs2bN08TJ05UTk6OrrvuOjU1NcWM6e7uVnl5ubKzs5Wenq4lS5aotbU1Zkxzc7PKysqUmpqqnJwc3Xvvverr6zv7RwMgholE1NnYqI5XX5WMcXTu1OnTNeXmm3k/FIbVoCJVU1Oj8vJyvfrqq6qqqlJvb68WL16s48ePR8fcc889euGFF7Rp0ybV1NTo0KFDuuGGG6L7+/v7VVZWpp6eHr3yyit68skntXHjRq1du9a5RwVAktTT1qZDv/iFeo8edXReV1KScsrK5MnN5SwKw8plzND/eXXkyBHl5OSopqZGl156qYLBoCZPnqynn35aN954oyTpnXfe0fnnn6/a2lotXLhQ27Zt09e+9jUdOnRIubm5kqTHH39c//Iv/6IjR44oeQB/VjoUCsnn8ykYDMrr9Q51+cCYFunpUdOaNTqxf7+zE7vdKvz2tzX5iisIFIZsoL/Hz+o1qeD/f7d6VlaWJKm+vl69vb0qKSmJjpkxY4aKiopUW1srSaqtrdWsWbOigZKk0tJShUIh7d2794zfJxwOKxQKxdwAfDLT16ejVVU68e67js89qaRE2Zdd5vi8wJkMOVKRSESrVq3Sl770Jc2cOVOSFAgElJycrIyMjJixubm5CgQC0TF/HahT+0/tO5OKigr5fL7orbCwcKjLBsaF0BtvKLBpkxSJODpvcl6esktKlJCaylkURsSQI1VeXq633npLlZWVTq7njNasWaNgMBi9tbS0DPv3BEYjY4xOvvee3v/JT9Tb3u7o3O7UVH3hvvv4XD6MqCF94sTKlSu1ZcsW7dy5UwUFBdHteXl56unpUUdHR8zZVGtrq/Ly8qJjdu3aFTPfqav/To35OI/HI4/HM5SlAuNKJBxW4LnnHA+UJBXcfrsmTJ3q+LzApxnUmZQxRitXrtTmzZu1fft2TZs2LWb/nDlzlJSUpOrq6ui2pqYmNTc3y+/3S5L8fr8aGxvV1tYWHVNVVSWv16vi4uKzeSzAuHdk61Z9+NJLjs/rvfhi+ebM4XP5MOIGdSZVXl6up59+Wr/5zW80ceLE6GtIPp9PEyZMkM/n0/Lly7V69WplZWXJ6/Xqrrvukt/v18KFCyVJixcvVnFxsW699VatX79egUBA999/v8rLyzlbAobIRCI6tn27DldWyvT3Ozp32nnnaeo99yjpY681AyNhUJHasGGDJOnyyy+P2f7EE0/on/7pnyRJDz/8sNxut5YsWaJwOKzS0lI99thj0bEJCQnasmWLVqxYIb/fr7S0NC1btkwPPfTQ2T0SYJwyxqj74EEd2bpVke5uZyd3uz/64Fifz9l5gQE6q/dJxQvvkwL+lzFGf6mo+OhTJRyWc+21Kli2TK5E/mACnDXQ3+P85AGjWKS3Vy0/+5k66uqcndjtVs7VV+tz3/gGgUJc8SooMEqZ/n4Fd+9WcPduxz+Xz3vJJcq78Ua5eZ0YcUakgFGqNxTSoV/+Ur3Hjjk7scul/K9/XUm8DgULcB4PjEK9H36o/Q88oO4PPnB0XndKis5ZuVKpn/+8o/MCQ8WZFDDK9Hd3q+VnP1N3c7Oj87qSkpRz7bXyzZ/P+6FgDX4SgVHEGKPOxkZ1NjY6PndSZqZyysqUkJLi+NzAUPF0HzCKHH/nHR1Yv16RcNjReZMyM/XFH/yAN+zCOpxJAaNEbzCoAw8/7HigEjMzVfDNbyo5J8fReQEnEClgFIj09urI1q3OX8knaeIFF8g3bx6vQ8FKPN0HWM709+vIli06/Oyzjv99qPQLLtDUe+6ROynJ0XkBpxApwHIn339fH2zc6PgbdlOKilS0YgWfKAGrcX4PWKyvq0sHn3zS8UDJ7dakkhKlfO5z/IVdWI1/QgGW6uvsVPNPf6rQnj3OTpyQoLzrr1fONdfwOhSsx08oYCETiejDl1/Whzt3Oj63b+5c5f393xMojAr8lAIW6j548KMLJRzm9niUt2SJ3LxhF6MET/cBljnZ0qK/rFun3vZ2R+dNmDhR53znO0o77zxeh8KowZkUYJH+EyfUunmzultanJ04IUG5pz6Xj0BhFCFSgCWMMep49VV9+NJLjs+dnJWlnK99jfdDYdTh6T7AEp0NDXr/0UdlensdnTelsFBf+N735J4wwdF5gZFApIA4M8aop61NB//7vx0PVFJmpvJvvVWeKVN4mg+jEk/3AXEWOXlSrZs368SBA47PnXb++cqYO5fLzTFqcSYFxJExRm2//a2ObN3q+NzeOXM07bvf5WOPMKrxzysgTowxOt7UpMCmTY7PnTp9ugpuu41AYdQjUkCchA8f1sGNGxXp7nZ0XldSkjL/9m+VUlDA61AY9fhnFhAnnW+8oa59+07bfvD4ce1pb1dnb68mp6TIP3my0gZx6XjejTcq7/rrnVwqEDdECoiDSG+veo4ejdlmjNGBri49uGeP3uvqUnd/v7xJSZqZman/M2+ekgZw8UPmpZcq97rrhmnVwMjj6T4gDnqPHVPrc8/FbPtLV5fu+OMf9XYwqJP9/TKSgr29+mNbm+6uq9Oxz3haMDEjQ5NLS/lcPowpRAqIA2OMTH9/zLYf7d2r4Ce8T2rX0aOqOnToE+dLmDhR55SXa+KsWbwOhTGFSAFjwJSbb5Zv/vx4LwNwHJECRrmUoiJlX345Z1AYk4gUYImywkIlfUJopqana3ZW1mnbU7/wBX3+3nuVOHHicC8PiAsiBcRBks+nrK98JWZbaX6+Hrz4YqUkJET/j5ngcinb49H/nTdPxRkZMeMTJk5UwfLlSikqGplFA3HAJehAHLgnTFCm36/ga6+pv7NTkuRyuVSan6+C1FRt+eADHevu1tT0dN00bZqyPZ7T5vBedJHSL7iAp/kwphEpIA5cLpe8F1+syVdeqdZf/zp6pZ/L5dLMzEzNzMz81PtnfvnLOqe8nEBhzCNSQJy4PR7lXn+9ej/8UO07dsj09X3mfRLS0zXlppuUvWiRElJTR2CVQHwRKSCOElJTVXD77UrKylL7jh3qaWs74zhXQoJSCguVe/31yuJKPowjRAqII5fLpcS0NE258UZ5Z8/Wh6+8oq69exUOBBQJh5WQnq6UggL55s6Vb+5cTSgqIlAYV4gUYAG3x6P0mTOV9sUvqv/kSZm+PplIRK6EBLmTkuROTZWbP7uBcYifesASLpdLLo9H7jNcyQeMV7xPCgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFqDitSGDRs0e/Zseb1eeb1e+f1+bdu2Lbq/u7tb5eXlys7OVnp6upYsWaLW1taYOZqbm1VWVqbU1FTl5OTo3nvvVV9fnzOPBgAwpgwqUgUFBVq3bp3q6+v12muv6atf/aquvfZa7d27V5J0zz336IUXXtCmTZtUU1OjQ4cO6YYbbojev7+/X2VlZerp6dErr7yiJ598Uhs3btTatWudfVQAgLHBnKXMzEzz85//3HR0dJikpCSzadOm6L63337bSDK1tbXGGGO2bt1q3G63CQQC0TEbNmwwXq/XhMPhAX/PYDBoJJlgMHi2ywcAxMFAf48P+TWp/v5+VVZW6vjx4/L7/aqvr1dvb69KSkqiY2bMmKGioiLV1tZKkmprazVr1izl5uZGx5SWlioUCkXPxs4kHA4rFArF3AAAY9+gI9XY2Kj09HR5PB7deeed2rx5s4qLixUIBJScnKyMjIyY8bm5uQoEApKkQCAQE6hT+0/t+yQVFRXy+XzRW2Fh4WCXDQAYhQYdqfPOO08NDQ2qq6vTihUrtGzZMu3bt2841ha1Zs0aBYPB6K2lpWVYvx8AwA6Jg71DcnKyzj33XEnSnDlztHv3bv34xz/WTTfdpJ6eHnV0dMScTbW2tiovL0+SlJeXp127dsXMd+rqv1NjzsTj8cjj8Qx2qQCAUe6s3ycViUQUDoc1Z84cJSUlqbq6OrqvqalJzc3N8vv9kiS/36/Gxka1tbVFx1RVVcnr9aq4uPhslwIAGGMGdSa1Zs0aXXnllSoqKlJnZ6eefvppvfjii/r9738vn8+n5cuXa/Xq1crKypLX69Vdd90lv9+vhQsXSpIWL16s4uJi3XrrrVq/fr0CgYDuv/9+lZeXc6YEADjNoCLV1tamf/zHf9Thw4fl8/k0e/Zs/f73v9ff/d3fSZIefvhhud1uLVmyROFwWKWlpXrsscei909ISNCWLVu0YsUK+f1+paWladmyZXrooYecfVQAgDHBZYwx8V7EYIVCIfl8PgWDQXm93ngvBwAwSAP9Pc5n9wEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCw1llFat26dXK5XFq1alV0W3d3t8rLy5Wdna309HQtWbJEra2tMfdrbm5WWVmZUlNTlZOTo3vvvVd9fX1nsxQAwBg05Ejt3r1bP/3pTzV79uyY7ffcc49eeOEFbdq0STU1NTp06JBuuOGG6P7+/n6VlZWpp6dHr7zyip588klt3LhRa9euHfqjAACMTWYIOjs7zfTp001VVZW57LLLzN13322MMaajo8MkJSWZTZs2Rce+/fbbRpKpra01xhizdetW43a7TSAQiI7ZsGGD8Xq9JhwOD+j7B4NBI8kEg8GhLB8AEGcD/T0+pDOp8vJylZWVqaSkJGZ7fX29ent7Y7bPmDFDRUVFqq2tlSTV1tZq1qxZys3NjY4pLS1VKBTS3r17z/j9wuGwQqFQzA0AMPYlDvYOlZWVev3117V79+7T9gUCASUnJysjIyNme25urgKBQHTMXwfq1P5T+86koqJC3//+9we7VADAKDeoM6mWlhbdfffdeuqpp5SSkjJcazrNmjVrFAwGo7eWlpYR+94AgPgZVKTq6+vV1tamSy65RImJiUpMTFRNTY0eeeQRJSYmKjc3Vz09Pero6Ii5X2trq/Ly8iRJeXl5p13td+rrU2M+zuPxyOv1xtwAAGPfoCK1aNEiNTY2qqGhIXqbO3euli5dGv3vpKQkVVdXR+/T1NSk5uZm+f1+SZLf71djY6Pa2tqiY6qqquT1elVcXOzQwwIAjAWDek1q4sSJmjlzZsy2tLQ0ZWdnR7cvX75cq1evVlZWlrxer+666y75/X4tXLhQkrR48WIVFxfr1ltv1fr16xUIBHT//fervLxcHo/HoYcFABgLBn3hxGd5+OGH5Xa7tWTJEoXDYZWWluqxxx6L7k9ISNCWLVu0YsUK+f1+paWladmyZXrooYecXgoAYJRzGWNMvBcxWKFQSD6fT8FgkNenAGAUGujvcT67DwBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgrcR4L2AojDGSpFAoFOeVAACG4tTv71O/zz/JqIzUsWPHJEmFhYVxXgkA4Gx0dnbK5/N94v5RGamsrCxJUnNz86c+uPEuFAqpsLBQLS0t8nq98V6OtThOA8NxGhiO08AYY9TZ2an8/PxPHTcqI+V2f/RSms/n44dgALxeL8dpADhOA8NxGhiO02cbyEkGF04AAKxFpAAA1hqVkfJ4PHrwwQfl8XjivRSrcZwGhuM0MByngeE4OctlPuv6PwAA4mRUnkkBAMYHIgUAsBaRAgBYi0gBAKw1KiP16KOPaurUqUpJSdGCBQu0a9eueC9pRO3cuVNXX3218vPz5XK59Pzzz8fsN8Zo7dq1mjJliiZMmKCSkhLt378/Zkx7e7uWLl0qr9erjIwMLV++XF1dXSP4KIZXRUWF5s2bp4kTJyonJ0fXXXedmpqaYsZ0d3ervLxc2dnZSk9P15IlS9Ta2hozprm5WWVlZUpNTVVOTo7uvfde9fX1jeRDGVYbNmzQ7Nmzo2889fv92rZtW3Q/x+jM1q1bJ5fLpVWrVkW3cayGiRllKisrTXJysvmv//ovs3fvXnPHHXeYjIwM09raGu+ljZitW7eaf/3XfzXPPfeckWQ2b94cs3/dunXG5/OZ559/3rzxxhvmmmuuMdOmTTMnT56MjrniiivMhRdeaF599VXz0ksvmXPPPdfccsstI/xIhk9paal54oknzFtvvWUaGhrMVVddZYqKikxXV1d0zJ133mkKCwtNdXW1ee2118zChQvN3/zN30T39/X1mZkzZ5qSkhKzZ88es3XrVjNp0iSzZs2aeDykYfE///M/5re//a3505/+ZJqamsz3vvc9k5SUZN566y1jDMfoTHbt2mWmTp1qZs+ebe6+++7odo7V8Bh1kZo/f74pLy+Pft3f32/y8/NNRUVFHFcVPx+PVCQSMXl5eeaHP/xhdFtHR4fxeDzmmWeeMcYYs2/fPiPJ7N69Ozpm27ZtxuVymYMHD47Y2kdSW1ubkWRqamqMMR8dk6SkJLNp06bomLfffttIMrW1tcaYj/4x4Ha7TSAQiI7ZsGGD8Xq9JhwOj+wDGEGZmZnm5z//OcfoDDo7O8306dNNVVWVueyyy6KR4lgNn1H1dF9PT4/q6+tVUlIS3eZ2u1VSUqLa2to4rsweBw4cUCAQiDlGPp9PCxYsiB6j2tpaZWRkaO7cudExJSUlcrvdqqurG/E1j4RgMCjpfz+cuL6+Xr29vTHHacaMGSoqKoo5TrNmzVJubm50TGlpqUKhkPbu3TuCqx8Z/f39qqys1PHjx+X3+zlGZ1BeXq6ysrKYYyLx8zScRtUHzB49elT9/f0x/yNLUm5urt555504rcougUBAks54jE7tCwQCysnJidmfmJiorKys6JixJBKJaNWqVfrSl76kmTNnSvroGCQnJysjIyNm7MeP05mO46l9Y0VjY6P8fr+6u7uVnp6uzZs3q7i4WA0NDRyjv1JZWanXX39du3fvPm0fP0/DZ1RFChiK8vJyvfXWW3r55ZfjvRQrnXfeeWpoaFAwGNSvfvUrLVu2TDU1NfFellVaWlp09913q6qqSikpKfFezrgyqp7umzRpkhISEk67Yqa1tVV5eXlxWpVdTh2HTztGeXl5amtri9nf19en9vb2MXccV65cqS1btmjHjh0qKCiIbs/Ly1NPT486Ojpixn/8OJ3pOJ7aN1YkJyfr3HPP1Zw5c1RRUaELL7xQP/7xjzlGf6W+vl5tbW265JJLlJiYqMTERNXU1OiRRx5RYmKicnNzOVbDZFRFKjk5WXPmzFF1dXV0WyQSUXV1tfx+fxxXZo9p06YpLy8v5hiFQiHV1dVFj5Hf71dHR4fq6+ujY7Zv365IJKIFCxaM+JqHgzFGK1eu1ObNm7V9+3ZNmzYtZv+cOXOUlJQUc5yamprU3Nwcc5waGxtjgl5VVSWv16vi4uKReSBxEIlEFA6HOUZ/ZdGiRWpsbFRDQ0P0NnfuXC1dujT63xyrYRLvKzcGq7Ky0ng8HrNx40azb98+861vfctkZGTEXDEz1nV2dpo9e/aYPXv2GEnm3//9382ePXvM+++/b4z56BL0jIwM85vf/Ma8+eab5tprrz3jJegXX3yxqaurMy+//LKZPn36mLoEfcWKFcbn85kXX3zRHD58OHo7ceJEdMydd95pioqKzPbt281rr71m/H6/8fv90f2nLhlevHixaWhoML/73e/M5MmTx9Qlw/fdd5+pqakxBw4cMG+++aa57777jMvlMn/4wx+MMRyjT/PXV/cZw7EaLqMuUsYY85Of/MQUFRWZ5ORkM3/+fPPqq6/Ge0kjaseOHUbSabdly5YZYz66DP2BBx4wubm5xuPxmEWLFpmmpqaYOY4dO2ZuueUWk56ebrxer7nttttMZ2dnHB7N8DjT8ZFknnjiieiYkydPmu985zsmMzPTpKammuuvv94cPnw4Zp733nvPXHnllWbChAlm0qRJ5rvf/a7p7e0d4UczfG6//XZzzjnnmOTkZDN58mSzaNGiaKCM4Rh9mo9HimM1PPhTHQAAa42q16QAAOMLkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANb6fxTFY2ufkAX9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 11])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "#DuelingDQN和其他DQN模型不同的点,它使用的是不同的模型结构\n",
    "class VAnet(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "        self.fc = torch.nn.Sequential(\n",
    "            torch.nn.Linear(3, 128),\n",
    "            torch.nn.ReLU(),\n",
    "        )\n",
    "\n",
    "        self.fc_A = torch.nn.Linear(128, 11)\n",
    "        self.fc_V = torch.nn.Linear(128, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        #[5, 11] -> [5, 128] -> [5, 11]\n",
    "        A = self.fc_A(self.fc(x))\n",
    "\n",
    "        #[5, 11] -> [5, 128] -> [5, 1]\n",
    "        V = self.fc_V(self.fc(x))\n",
    "\n",
    "        #[5, 11] -> [5] -> [5, 1]\n",
    "        A_mean = A.mean(dim=1).reshape(-1, 1)\n",
    "\n",
    "        #[5, 11] - [5, 1] = [5, 11]\n",
    "        A -= A_mean\n",
    "\n",
    "        #Q值由V值和A值计算得到\n",
    "        #[5, 11] + [5, 1] = [5, 11]\n",
    "        Q = A + V\n",
    "\n",
    "        return Q\n",
    "\n",
    "\n",
    "VAnet()(torch.randn(5, 3)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(VAnet(\n",
       "   (fc): Sequential(\n",
       "     (0): Linear(in_features=3, out_features=128, bias=True)\n",
       "     (1): ReLU()\n",
       "   )\n",
       "   (fc_A): Linear(in_features=128, out_features=11, bias=True)\n",
       "   (fc_V): Linear(in_features=128, out_features=1, bias=True)\n",
       " ),\n",
       " VAnet(\n",
       "   (fc): Sequential(\n",
       "     (0): Linear(in_features=3, out_features=128, bias=True)\n",
       "     (1): ReLU()\n",
       "   )\n",
       "   (fc_A): Linear(in_features=128, out_features=11, bias=True)\n",
       "   (fc_V): Linear(in_features=128, out_features=1, bias=True)\n",
       " ))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "#计算动作的模型,也是真正要用的模型\n",
    "model = VAnet()\n",
    "\n",
    "#经验网络,用于评估一个状态的分数\n",
    "next_model = VAnet()\n",
    "\n",
    "#把model的参数复制给next_model\n",
    "next_model.load_state_dict(model.state_dict())\n",
    "\n",
    "model, next_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, -0.8)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "\n",
    "def get_action(state):\n",
    "    #走神经网络,得到一个动作\n",
    "    state = torch.FloatTensor(state).reshape(1, 3)\n",
    "    action = model(state).argmax().item()\n",
    "\n",
    "    if random.random() < 0.01:\n",
    "        action = random.choice(range(11))\n",
    "\n",
    "    #离散动作连续化\n",
    "    action_continuous = action\n",
    "    action_continuous /= 10\n",
    "    action_continuous *= 4\n",
    "    action_continuous -= 2\n",
    "\n",
    "    return action, action_continuous\n",
    "\n",
    "\n",
    "get_action([0.29292667, 0.9561349, 1.0957013])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((200, 0), 200)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#样本池\n",
    "datas = []\n",
    "\n",
    "\n",
    "#向样本池中添加N条数据,删除M条最古老的数据\n",
    "def update_data():\n",
    "    old_count = len(datas)\n",
    "\n",
    "    #玩到新增了N个数据为止\n",
    "    while len(datas) - old_count < 200:\n",
    "        #初始化游戏\n",
    "        state = env.reset()\n",
    "\n",
    "        #玩到游戏结束为止\n",
    "        over = False\n",
    "        while not over:\n",
    "            #根据当前状态得到一个动作\n",
    "            action, action_continuous = get_action(state)\n",
    "\n",
    "            #执行动作,得到反馈\n",
    "            next_state, reward, over, _ = env.step([action_continuous])\n",
    "\n",
    "            #记录数据样本\n",
    "            datas.append((state, action, reward, next_state, over))\n",
    "\n",
    "            #更新游戏状态,开始下一个动作\n",
    "            state = next_state\n",
    "\n",
    "    update_count = len(datas) - old_count\n",
    "    drop_count = max(len(datas) - 5000, 0)\n",
    "\n",
    "    #数据上限,超出时从最古老的开始删除\n",
    "    while len(datas) > 5000:\n",
    "        datas.pop(0)\n",
    "\n",
    "    return update_count, drop_count\n",
    "\n",
    "\n",
    "update_data(), len(datas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1456/1416897299.py:7: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 3)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[ 0.9759,  0.2183, -0.8936],\n",
       "         [-0.4725,  0.8813, -3.0240],\n",
       "         [-0.1743, -0.9847,  0.6815],\n",
       "         [ 0.9895,  0.1446,  0.0487],\n",
       "         [ 0.9373,  0.3485,  4.5409],\n",
       "         [-0.9973, -0.0729, -5.6901],\n",
       "         [ 0.7743,  0.6328,  5.1138],\n",
       "         [-0.8478,  0.5303,  4.0918],\n",
       "         [ 0.8918,  0.4524, -1.6551],\n",
       "         [-0.2661, -0.9640,  7.2406],\n",
       "         [-0.3598,  0.9330, -2.4830],\n",
       "         [ 0.7067, -0.7075,  5.0712],\n",
       "         [-0.9910, -0.1336,  8.0000],\n",
       "         [-0.8945, -0.4470,  8.0000],\n",
       "         [-0.3082, -0.9513,  7.2930],\n",
       "         [-0.4207,  0.9072,  8.0000],\n",
       "         [-0.1909,  0.9816,  5.9905],\n",
       "         [-0.9967, -0.0811, -4.1083],\n",
       "         [ 0.7361,  0.6769, -2.2856],\n",
       "         [-0.4541,  0.8910,  2.4770],\n",
       "         [-0.4730, -0.8811,  3.0176],\n",
       "         [ 0.9875,  0.1576,  4.3611],\n",
       "         [ 0.8510, -0.5252,  4.6605],\n",
       "         [-0.2706, -0.9627,  1.8972],\n",
       "         [ 0.7631, -0.6463,  4.9572],\n",
       "         [-0.9439,  0.3302,  8.0000],\n",
       "         [ 0.6820,  0.7314,  3.0031],\n",
       "         [ 0.7837,  0.6211,  2.4772],\n",
       "         [ 0.9026,  0.4304,  1.6447],\n",
       "         [-0.7831, -0.6219,  3.0195],\n",
       "         [-0.4836,  0.8753,  8.0000],\n",
       "         [-0.9745,  0.2243, -5.9848],\n",
       "         [-0.5549, -0.8319,  0.5825],\n",
       "         [ 0.3246,  0.9459,  6.4068],\n",
       "         [-0.3898, -0.9209,  7.4036],\n",
       "         [-0.6272, -0.7789,  7.7572],\n",
       "         [-0.9874, -0.1585,  3.9467],\n",
       "         [-0.9720,  0.2348,  3.8619],\n",
       "         [-0.9395, -0.3424,  4.4839],\n",
       "         [-0.7121, -0.7021, -2.0979],\n",
       "         [-0.6786,  0.7345, -5.9181],\n",
       "         [-0.9436,  0.3311,  4.4295],\n",
       "         [-0.1712, -0.9852,  0.0630],\n",
       "         [ 0.5395,  0.8420,  3.6116],\n",
       "         [ 0.9826,  0.1855, -0.6699],\n",
       "         [-0.1963,  0.9805,  0.5509],\n",
       "         [ 0.9917,  0.1285,  4.3245],\n",
       "         [-0.1726,  0.9850, -0.6871],\n",
       "         [-0.8015, -0.5981, -2.7445],\n",
       "         [-0.4001,  0.9165,  0.4845],\n",
       "         [-0.7408,  0.6717,  8.0000],\n",
       "         [ 0.6541,  0.7564, -2.6129],\n",
       "         [ 0.9846,  0.1748,  0.3938],\n",
       "         [-0.9246,  0.3810,  8.0000],\n",
       "         [-0.6997, -0.7144,  2.4931],\n",
       "         [ 0.9092,  0.4164,  4.6815],\n",
       "         [ 0.0201,  0.9998,  7.1613],\n",
       "         [-0.3405,  0.9402,  1.8318],\n",
       "         [-0.7863,  0.6178,  8.0000],\n",
       "         [-0.2829, -0.9591,  7.2611],\n",
       "         [ 0.1073,  0.9942,  5.0648],\n",
       "         [-0.5119,  0.8590,  6.9067],\n",
       "         [ 0.9962, -0.0872,  4.2699],\n",
       "         [ 0.5639, -0.8258,  5.4790]]),\n",
       " tensor([[6],\n",
       "         [3],\n",
       "         [7],\n",
       "         [6],\n",
       "         [7],\n",
       "         [1],\n",
       "         [8],\n",
       "         [4],\n",
       "         [2],\n",
       "         [7],\n",
       "         [3],\n",
       "         [7],\n",
       "         [7],\n",
       "         [7],\n",
       "         [7],\n",
       "         [8],\n",
       "         [8],\n",
       "         [3],\n",
       "         [1],\n",
       "         [4],\n",
       "         [7],\n",
       "         [7],\n",
       "         [7],\n",
       "         [7],\n",
       "         [7],\n",
       "         [8],\n",
       "         [6],\n",
       "         [6],\n",
       "         [6],\n",
       "         [4],\n",
       "         [8],\n",
       "         [1],\n",
       "         [7],\n",
       "         [8],\n",
       "         [7],\n",
       "         [7],\n",
       "         [4],\n",
       "         [4],\n",
       "         [4],\n",
       "         [3],\n",
       "         [1],\n",
       "         [4],\n",
       "         [7],\n",
       "         [4],\n",
       "         [6],\n",
       "         [3],\n",
       "         [7],\n",
       "         [3],\n",
       "         [3],\n",
       "         [3],\n",
       "         [8],\n",
       "         [1],\n",
       "         [6],\n",
       "         [8],\n",
       "         [4],\n",
       "         [7],\n",
       "         [8],\n",
       "         [4],\n",
       "         [8],\n",
       "         [7],\n",
       "         [8],\n",
       "         [8],\n",
       "         [7],\n",
       "         [7]]),\n",
       " tensor([[ -0.1285],\n",
       "         [ -5.1709],\n",
       "         [ -3.0954],\n",
       "         [ -0.0215],\n",
       "         [ -2.1893],\n",
       "         [-12.6566],\n",
       "         [ -3.0860],\n",
       "         [ -8.3445],\n",
       "         [ -0.4957],\n",
       "         [ -8.6292],\n",
       "         [ -4.3762],\n",
       "         [ -3.1900],\n",
       "         [-15.4463],\n",
       "         [-13.5731],\n",
       "         [ -8.8693],\n",
       "         [-10.4216],\n",
       "         [ -6.6977],\n",
       "         [-11.0545],\n",
       "         [ -1.0778],\n",
       "         [ -4.7840],\n",
       "         [ -5.1692],\n",
       "         [ -1.9276],\n",
       "         [ -2.4784],\n",
       "         [ -3.7638],\n",
       "         [ -2.9518],\n",
       "         [-14.2697],\n",
       "         [ -1.5750],\n",
       "         [ -1.0630],\n",
       "         [ -0.4687],\n",
       "         [ -7.0148],\n",
       "         [-10.7095],\n",
       "         [-12.0837],\n",
       "         [ -4.6962],\n",
       "         [ -5.6443],\n",
       "         [ -9.3677],\n",
       "         [-11.0749],\n",
       "         [-10.4524],\n",
       "         [ -9.9281],\n",
       "         [ -9.8065],\n",
       "         [ -6.0259],\n",
       "         [ -8.8716],\n",
       "         [ -9.8253],\n",
       "         [ -3.0384],\n",
       "         [ -2.3064],\n",
       "         [ -0.0799],\n",
       "         [ -3.1582],\n",
       "         [ -1.8874],\n",
       "         [ -3.0903],\n",
       "         [ -7.0065],\n",
       "         [ -3.9540],\n",
       "         [-12.1856],\n",
       "         [ -1.4211],\n",
       "         [ -0.0465],\n",
       "         [-13.9677],\n",
       "         [ -6.1243],\n",
       "         [ -2.3767],\n",
       "         [ -7.5345],\n",
       "         [ -4.0156],\n",
       "         [-12.5300],\n",
       "         [ -8.7238],\n",
       "         [ -4.7080],\n",
       "         [ -9.2162],\n",
       "         [ -1.8315],\n",
       "         [ -3.9468]]),\n",
       " tensor([[ 0.9826,  0.1855, -0.6699],\n",
       "         [-0.3598,  0.9330, -2.4830],\n",
       "         [-0.1712, -0.9852,  0.0630],\n",
       "         [ 0.9879,  0.1554,  0.2172],\n",
       "         [ 0.8242,  0.5663,  4.9223],\n",
       "         [-0.9745,  0.2243, -5.9848],\n",
       "         [ 0.5623,  0.8269,  5.7684],\n",
       "         [-0.9436,  0.3311,  4.4295],\n",
       "         [ 0.9231,  0.3845, -1.4958],\n",
       "         [ 0.0625, -0.9980,  6.6376],\n",
       "         [-0.2695,  0.9630, -1.9032],\n",
       "         [ 0.8510, -0.5252,  4.6605],\n",
       "         [-0.8608, -0.5090,  8.0000],\n",
       "         [-0.6580, -0.7530,  7.7847],\n",
       "         [ 0.0217, -0.9998,  6.6995],\n",
       "         [-0.7408,  0.6717,  8.0000],\n",
       "         [-0.5119,  0.8590,  6.9067],\n",
       "         [-0.9911,  0.1329, -4.2891],\n",
       "         [ 0.8005,  0.5993, -2.0180],\n",
       "         [-0.5856,  0.8106,  3.0852],\n",
       "         [-0.3605, -0.9327,  2.4768],\n",
       "         [ 0.9256,  0.3785,  4.5993],\n",
       "         [ 0.9449, -0.3274,  4.3867],\n",
       "         [-0.2077, -0.9782,  1.2952],\n",
       "         [ 0.8902, -0.4556,  4.5925],\n",
       "         [-0.9980, -0.0634,  8.0000],\n",
       "         [ 0.5395,  0.8420,  3.6116],\n",
       "         [ 0.6820,  0.7314,  3.0031],\n",
       "         [ 0.8544,  0.5196,  2.0276],\n",
       "         [-0.6997, -0.7144,  2.4931],\n",
       "         [-0.7863,  0.6178,  8.0000],\n",
       "         [-0.8633,  0.5047, -6.0566],\n",
       "         [-0.5516, -0.8341,  0.0786],\n",
       "         [-0.0342,  0.9994,  7.2962],\n",
       "         [-0.0588, -0.9983,  6.8329],\n",
       "         [-0.3082, -0.9513,  7.2930],\n",
       "         [-0.9402, -0.3406,  3.7678],\n",
       "         [-0.9993,  0.0381,  3.9781],\n",
       "         [-0.8484, -0.5294,  4.1671],\n",
       "         [-0.8015, -0.5981, -2.7445],\n",
       "         [-0.4488,  0.8936, -5.6072],\n",
       "         [-0.9943,  0.1064,  4.6178],\n",
       "         [-0.1985, -0.9801, -0.5559],\n",
       "         [ 0.3529,  0.9356,  4.1831],\n",
       "         [ 0.9867,  0.1623, -0.4707],\n",
       "         [-0.2531,  0.9674,  1.1663],\n",
       "         [ 0.9373,  0.3485,  4.5409],\n",
       "         [-0.1692,  0.9856, -0.0683],\n",
       "         [-0.8891, -0.4577, -3.3130],\n",
       "         [-0.4477,  0.8942,  1.0519],\n",
       "         [-0.9439,  0.3302,  8.0000],\n",
       "         [ 0.7361,  0.6769, -2.2856],\n",
       "         [ 0.9791,  0.2035,  0.5849],\n",
       "         [-1.0000, -0.0091,  8.0000],\n",
       "         [-0.6289, -0.7775,  1.8972],\n",
       "         [ 0.7743,  0.6328,  5.1138],\n",
       "         [-0.3708,  0.9287,  8.0000],\n",
       "         [-0.4541,  0.8910,  2.4770],\n",
       "         [-0.9648,  0.2629,  8.0000],\n",
       "         [ 0.0462, -0.9989,  6.6618],\n",
       "         [-0.1909,  0.9816,  5.9905],\n",
       "         [-0.7980,  0.6027,  7.7310],\n",
       "         [ 0.9917,  0.1285,  4.3245],\n",
       "         [ 0.7500, -0.6614,  4.9796]]),\n",
       " tensor([[0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0]]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取一批数据样本\n",
    "def get_sample():\n",
    "    #从样本池中采样\n",
    "    samples = random.sample(datas, 64)\n",
    "\n",
    "    #[b, 3]\n",
    "    state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 3)\n",
    "    #[b, 1]\n",
    "    action = torch.LongTensor([i[1] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 1]\n",
    "    reward = torch.FloatTensor([i[2] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 3]\n",
    "    next_state = torch.FloatTensor([i[3] for i in samples]).reshape(-1, 3)\n",
    "    #[b, 1]\n",
    "    over = torch.LongTensor([i[4] for i in samples]).reshape(-1, 1)\n",
    "\n",
    "    return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "state, action, reward, next_state, over"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.2186],\n",
       "        [1.0698],\n",
       "        [0.3330],\n",
       "        [0.3346],\n",
       "        [1.6194],\n",
       "        [1.4954],\n",
       "        [1.8757],\n",
       "        [1.6286],\n",
       "        [0.0499],\n",
       "        [2.4810],\n",
       "        [0.9849],\n",
       "        [1.8553],\n",
       "        [2.6182],\n",
       "        [2.6429],\n",
       "        [2.4936],\n",
       "        [3.1149],\n",
       "        [2.3909],\n",
       "        [1.1692],\n",
       "        [0.5405],\n",
       "        [1.3310],\n",
       "        [0.8825],\n",
       "        [1.5797],\n",
       "        [1.7221],\n",
       "        [0.5553],\n",
       "        [1.8204],\n",
       "        [2.8339],\n",
       "        [1.2302],\n",
       "        [1.0705],\n",
       "        [0.8048],\n",
       "        [0.8547],\n",
       "        [3.1013],\n",
       "        [1.5785],\n",
       "        [0.2799],\n",
       "        [2.5150],\n",
       "        [2.5211],\n",
       "        [2.6108],\n",
       "        [1.2935],\n",
       "        [1.4403],\n",
       "        [1.3595],\n",
       "        [0.6939],\n",
       "        [1.5672],\n",
       "        [1.6380],\n",
       "        [0.2888],\n",
       "        [1.4279],\n",
       "        [0.2320],\n",
       "        [0.8344],\n",
       "        [1.5701],\n",
       "        [0.7657],\n",
       "        [0.8718],\n",
       "        [0.8129],\n",
       "        [3.0086],\n",
       "        [0.6486],\n",
       "        [0.3951],\n",
       "        [2.8619],\n",
       "        [0.6737],\n",
       "        [1.6606],\n",
       "        [2.8338],\n",
       "        [1.1485],\n",
       "        [2.9832],\n",
       "        [2.4859],\n",
       "        [2.0468],\n",
       "        [2.6806],\n",
       "        [1.5734],\n",
       "        [1.9822]], grad_fn=<GatherBackward0>)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_value(state, action):\n",
    "    #使用状态计算出动作的logits\n",
    "    #[b, 3] -> [b, 11]\n",
    "    value = model(state)\n",
    "\n",
    "    #根据实际使用的action取出每一个值\n",
    "    #这个值就是模型评估的在该状态下,执行动作的分数\n",
    "    #在执行动作前,显然并不知道会得到的反馈和next_state\n",
    "    #所以这里不能也不需要考虑next_state和reward\n",
    "    #[b, 11] -> [b, 1]\n",
    "    value = value.gather(dim=1, index=action)\n",
    "\n",
    "    return value\n",
    "\n",
    "\n",
    "get_value(state, action)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[  0.0989],\n",
       "        [ -4.2057],\n",
       "        [ -2.8124],\n",
       "        [  0.3336],\n",
       "        [ -0.4502],\n",
       "        [-11.1096],\n",
       "        [ -0.9207],\n",
       "        [ -6.7393],\n",
       "        [ -0.2133],\n",
       "        [ -6.3599],\n",
       "        [ -3.5263],\n",
       "        [ -1.5024],\n",
       "        [-12.8474],\n",
       "        [-11.0097],\n",
       "        [ -6.5850],\n",
       "        [ -7.4731],\n",
       "        [ -4.0707],\n",
       "        [ -9.8589],\n",
       "        [ -0.6378],\n",
       "        [ -3.3438],\n",
       "        [ -4.4678],\n",
       "        [ -0.3240],\n",
       "        [ -0.8849],\n",
       "        [ -3.3451],\n",
       "        [ -1.2864],\n",
       "        [-11.7006],\n",
       "        [ -0.1756],\n",
       "        [  0.1426],\n",
       "        [  0.4434],\n",
       "        [ -6.3546],\n",
       "        [ -7.7860],\n",
       "        [-10.5221],\n",
       "        [ -4.4468],\n",
       "        [ -2.8167],\n",
       "        [ -7.0503],\n",
       "        [ -8.6312],\n",
       "        [ -9.3022],\n",
       "        [ -8.5745],\n",
       "        [ -8.5947],\n",
       "        [ -5.1716],\n",
       "        [ -7.4085],\n",
       "        [ -8.2715],\n",
       "        [ -2.7818],\n",
       "        [ -0.6612],\n",
       "        [  0.1618],\n",
       "        [ -2.2332],\n",
       "        [ -0.3004],\n",
       "        [ -2.3261],\n",
       "        [ -6.0247],\n",
       "        [ -3.1003],\n",
       "        [ -9.4084],\n",
       "        [ -0.8914],\n",
       "        [  0.3700],\n",
       "        [-11.3705],\n",
       "        [ -5.6301],\n",
       "        [ -0.5386],\n",
       "        [ -4.4736],\n",
       "        [ -2.7111],\n",
       "        [ -9.7893],\n",
       "        [ -6.4487],\n",
       "        [ -2.3650],\n",
       "        [ -6.3986],\n",
       "        [ -0.2928],\n",
       "        [ -2.1561]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_target(reward, next_state, over):\n",
    "    #上面已经把模型认为的状态下执行动作的分数给评估出来了\n",
    "    #下面使用next_state和reward计算真实的分数\n",
    "    #针对一个状态,它到底应该多少分,可以使用以往模型积累的经验评估\n",
    "    #这也是没办法的办法,因为显然没有精确解,这里使用延迟更新的next_model评估\n",
    "\n",
    "    #使用next_state计算下一个状态的分数\n",
    "    #[b, 3] -> [b, 11]\n",
    "    with torch.no_grad():\n",
    "        target = next_model(next_state)\n",
    "\n",
    "    #取所有动作中分数最大的\n",
    "    #[b, 11] -> [b, 1]\n",
    "    target = target.max(dim=1)[0]\n",
    "    target = target.reshape(-1, 1)\n",
    "\n",
    "    #下一个状态的分数乘以一个系数,相当于权重\n",
    "    target *= 0.98\n",
    "\n",
    "    #如果next_state已经游戏结束,则next_state的分数是0\n",
    "    #因为如果下一步已经游戏结束,显然不需要再继续玩下去,也就不需要考虑next_state了.\n",
    "    #[b, 1] * [b, 1] -> [b, 1]\n",
    "    target *= (1 - over)\n",
    "\n",
    "    #加上reward就是最终的分数\n",
    "    #[b, 1] + [b, 1] -> [b, 1]\n",
    "    target += reward\n",
    "\n",
    "    return target\n",
    "\n",
    "\n",
    "get_target(reward, next_state, over)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1782.7036413962624"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "def test(play):\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #记录反馈值的和,这个值越大越好\n",
    "    reward_sum = 0\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        _, action_continuous = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        state, reward, over, _ = env.step([action_continuous])\n",
    "        reward_sum += reward\n",
    "\n",
    "        #打印动画\n",
    "        if play and random.random() < 0.2:  #跳帧\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    return reward_sum\n",
    "\n",
    "\n",
    "test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "OHoSU6uI-xIt",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 400 200 0 -1549.6536307654428\n",
      "20 4400 200 0 -774.4212962756064\n",
      "40 5000 200 200 -544.4996603817052\n",
      "60 5000 200 200 -257.0068187439457\n",
      "80 5000 200 200 -548.199814393714\n",
      "100 5000 200 200 -335.9420079992984\n",
      "120 5000 200 200 -168.05497405405734\n",
      "140 5000 200 200 -408.53733198713195\n",
      "160 5000 200 200 -412.63491065532145\n",
      "180 5000 200 200 -296.082461949791\n"
     ]
    }
   ],
   "source": [
    "def train():\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=1e-2)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #训练N次\n",
    "    for epoch in range(200):\n",
    "        #更新N条数据\n",
    "        update_count, drop_count = update_data()\n",
    "\n",
    "        #每次更新过数据后,学习N次\n",
    "        for i in range(200):\n",
    "            #采样一批数据\n",
    "            state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "            #计算一批样本的value和target\n",
    "            value = get_value(state, action)\n",
    "            target = get_target(reward, next_state, over)\n",
    "\n",
    "            #更新参数\n",
    "            loss = loss_fn(value, target)\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            #把model的参数复制给next_model\n",
    "            if (i + 1) % 50 == 0:\n",
    "                next_model.load_state_dict(model.state_dict())\n",
    "\n",
    "        if epoch % 20 == 0:\n",
    "            test_result = sum([test(play=False) for _ in range(20)]) / 20\n",
    "            print(epoch, len(datas), update_count, drop_count, test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyrklEQVR4nO3deXxTZb4/8M85aZIuIeneWGmhCIK1FGXP6LhRKVpBZzqOetmuol6Y4qjcy+uKCvOS66Uo4wJXFme8iuOM4MAFZBWZglWhbJVCgVIXlpZCWkpp0jVtk+f3B5IfRXBamuUJ+bxfr7xe5pwn53zzWPLJOefJcxQhhAAREZGEVH8XQEREdCUMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpKW30Jq4cKF6NmzJ0JDQzFs2DDs3r3bX6UQEZGk/BJSn3zyCaZNm4Y//OEP+OabbzBgwABkZmaiqqrKH+UQEZGkFH9MMDts2DAMGTIE77zzDgDA5XIhKSkJzzzzDF544QVfl0NERJIK8fUOW1paUFhYiBkzZriXqaqKjIwMFBQUXPY1DocDDofD/dzlcqGmpgYxMTFQFMXrNRMRkWcJIVBXV4fExESo6pVP6vk8pKqrq+F0OpGQkNBueUJCAo4cOXLZ1+Tm5uKVV17xRXlERORD5eXl6N69+xXX+zykrsaMGTMwbdo093ObzYbk5GSUl5fDaDT6sTIiIroadrsdSUlJ6Nat28+283lIxcbGQqPRoLKyst3yyspKmM3my75Gr9dDr9f/ZLnRaGRIEREFsH92ycbno/t0Oh0GDRqEvLw89zKXy4W8vDxYLBZfl0NERBLzy+m+adOmYeLEiRg8eDCGDh2Kt99+Gw0NDXj88cf9UQ4REUnKLyH1yCOP4MyZM5g1axasVituueUWfPbZZz8ZTEFERMHNL7+T6iq73Q6TyQSbzcZrUkREAaijn+Ocu4+IiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFqdDqkvv/wSo0ePRmJiIhRFwZo1a9qtF0Jg1qxZuO666xAWFoaMjAx899137drU1NRg7NixMBqNiIyMxKRJk1BfX9+lN0JERNeeTodUQ0MDBgwYgIULF152/euvv44FCxZgyZIl2LVrFyIiIpCZmYnm5mZ3m7Fjx+LQoUPYsmUL1q9fjy+//BJPP/301b8LIiK6NokuACBWr17tfu5yuYTZbBbz5s1zL6utrRV6vV4sW7ZMCCHE4cOHBQCxZ88ed5tNmzYJRVFERUVFh/Zrs9kEAGGz2bpSPhER+UlHP8c9ek3q2LFjsFqtyMjIcC8zmUwYNmwYCgoKAAAFBQWIjIzE4MGD3W0yMjKgqip27dp12e06HA7Y7fZ2DyIiuvZ5NKSsVisAICEhod3yhIQE9zqr1Yr4+Ph260NCQhAdHe1uc6nc3FyYTCb3IykpyZNlExGRpAJidN+MGTNgs9ncj/Lycn+XREREPuDRkDKbzQCAysrKdssrKyvd68xmM6qqqtqtb2trQ01NjbvNpfR6PYxGY7sHERFd+zwaUikpKTCbzcjLy3Mvs9vt2LVrFywWCwDAYrGgtrYWhYWF7jZbt26Fy+XCsGHDPFkOEREFuJDOvqC+vh7ff/+9+/mxY8dQVFSE6OhoJCcn47nnnsOrr76KPn36ICUlBTNnzkRiYiIeeughAMBNN92EUaNG4amnnsKSJUvQ2tqKqVOn4tFHH0ViYqLH3hgREV0DOjtscNu2bQLATx4TJ04UQpwfhj5z5kyRkJAg9Hq9GDFihCgtLW23jbNnz4rHHntMGAwGYTQaxeOPPy7q6uo8PnSRiIjk1NHPcUUIIfyYkVfFbrfDZDLBZrPx+hQRUQDq6Od4QIzuIyKi4MSQIiIiaTGkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFqdngWdiDpOCAE4nXBUVqJ21y7UHzmCNpsNqk6H0O7dYbzlFnRLT4eq10NR+Z2R6FIMKSIvaqmsRNX69Ti7bRuc9fXARfM51+3fjzObNiGiTx+YH34Y3QYMgEav92O1RPLhVzciL2k6cQJlf/oTqtauhbOurl1AublcaCgtRdmiRajevBnC6fR9oUQSY0gReZgQAg6rFSc//BD2b77p0Gtaa2pwetkynP3iCwiXy8sVEgUOhhSRhwmnE9YVK2DfuxfoROA4Gxpw6m9/Q31JiRerIwosDCkiD3NYrajeuvWqXttaXY1zX30Fl8Ph4aqIAhNDisjDKj74AOjCtaWz27ah6fhxzxVEFMAYUkQe1ma3d+n1rqYmOBsbzw9fJwpyDCkiCbXU1Pi7BCIpMKSIJNRy5szlh6wTBRmGFJGE6g8d6tTIQKJrFUOKSEL1JSX8vRQRGFJEHqcxGPxdAtE1gyFF5GHRd90FeGCyWI7uI2JIEXmcNiqq6xsRAm11dV3fDlGAY0gReZguNhZQlK5tRAi0VFV5piCiAMaQIvKwEJMJXYwoQAi0nj3riXKIAhpDisjDPHHzQuF0onbnTg9UQxTYGFJEkmqrrfV3CUR+x5AiIiJpMaSIPEwJCUHUL3/Z5e0IpxOulhYPVEQUuBhSRB6mqCrCevTo8nZcDgeHoVPQY0gReZqiIMQDv5VytbTAWV/vgYKIAhdDisjTFAVao7HLm3GcPo26Awc8UBBR4GJIEXmYoiiemRaprQ3O5mYPVEQUuBhSREQkLYYUkRcoGg0UrbbrGxKCE81SUGNIEXlBaGIiDKmpXd5OW10dRGurByoiCkwMKSIvUPV6aCIiurydNrudIUVBjSFF5AWqXu+Rmx+2nDnDwRMU1BhSRF6g6HTQhIV1eTv1Bw+i9dw5D1REFJgYUkReoChK1+8pRUQMKSLpcYQfBTGGFJGX6BMSoOr1Xd6Oq6nJA9UQBSaGFJGXhPfq5ZERfi3V1R6ohigwMaSIvCTEZPLID3odVVUeqIYoMDGkiLwkxGiE6oGQqjtwAOA1KQpSDCkiL9GEh0PRaLq8ncajRxlSFLQYUkReonhgJnSiYMd/RUQBQLS1+bsEIr9gSBF5kaLTdX0jLhdaa2u7vh2iAMSQIvKiuKysLm9DCMFh6BS0GFJEXqSLju7yNkRrK2x793qgGqLAw5Ai8iJtTEzX5/BzudBcVuaZgogCDEOKyItCTCZ/l0AU0BhSRF6keHAmdE4yS8GIIUUUAFwtLXA5HP4ug8jnGFJEXqSoKvTXXdfl7biam+Gsr/dARUSBhSFF5EWKVotu6eld3o6zuRltDQ0eqIgosIT4uwCia5miqtB2YRh6U1sbKpub4Tp8GLbPP0fcXXdBp9NBp9MhLCwMYWFh0Ol0Hr32RSQThhSRN6kqtFFRV/3yEpsNM7/5BmcdDoQWFMAYFYWoqCjExcUhKSkJPXv2RM+ePXHjjTfihhtuQFxcHEJC+M+arh38aybyJkXp0o0PNYoCFedH9jU5HGi0WmG1WlFSUgIAUFUVYWFhSEhIQFJSEgYMGIAHHngAQ4YMgcFggKIoPMqigNapa1K5ubkYMmQIunXrhvj4eDz00EMoLS1t16a5uRk5OTmIiYmBwWBAdnY2Kisr27UpKytDVlYWwsPDER8fj+nTp6ONE2jSNUhRlC7Nhp4QGopHe/XCxN698URmJsaPG4eMjAwMGjQIffr0QVxcHIQQOHr0KPLz8/HOO+/gwQcfxB133IFXX30VxcXFsNlsHL5OAatTR1L5+fnIycnBkCFD0NbWhhdffBEjR47E4cOHEfHjt8Xnn38eGzZswIoVK2AymTB16lT8+te/xvbt2wEATqcTWVlZMJvN2LFjB06fPo0JEyZAq9Vizpw5nn+HRAHMHB6OsTfcAABI+PWvkfgv/wKEhKC+vh4VFRU4duwYfvjhBxw8eBAHDhzAt99+i9raWhQXF6O4uBgffPABMjMzMWbMGNx5553uf6dEgUIRXfiKdebMGcTHxyM/Px933HEHbDYb4uLi8PHHH+M3v/kNAODIkSO46aabUFBQgOHDh2PTpk144IEHcOrUKSQkJAAAlixZgv/8z//EmTNnoOvArNF2ux0mkwk2mw1Go/FqyyfyiXMFBTg6d26Xb1wYPWIEkp96Cprw8HbLXS4XGhsbUVVVhWPHjmHz5s1Yt24dTpw4AYfDAVVVYTabce+992Ly5Mm45ZZboNVqeRqQ/Kqjn+NdGoJus9kAANE/jl4qLCxEa2srMjIy3G369euH5ORkFBQUAAAKCgrQv39/d0ABQGZmJux2Ow4dOnTZ/TgcDtjt9nYPokARnpICQ2pql7fjOHkSzubmnyxXVRUGgwEpKSm45557MGfOHHz11Vd4//33MWLECERHR+P06dP4y1/+gjFjxuCVV17B8ePH4XQ6u1wTkbdddUi5XC4899xzuO2225CWlgYAsFqt0Ol0iIyMbNc2ISEBVqvV3ebigLqw/sK6y8nNzYXJZHI/kpKSrrZsIp9Tw8K6NHjigobSUjh/5rdSFwZJhISEIDY2Fo888giWLVuG+fPnY/To0dDr9Thz5gzmzZuHJ598EmvWrIHD4eD1KpLaVYdUTk4ODh48iOXLl3uynsuaMWMGbDab+1FeXu71fRJ5iiY01CMh1VmKoiAmJga//e1vsWTJEsyfPx9paWlQFAX5+fmYOnUq5syZA7vdzqAiaV1VSE2dOhXr16/Htm3b0L17d/dys9mMlpYW1F5yF9HKykqYzWZ3m0tH+114fqHNpfR6PYxGY7sHUaBQdDqoer1nNuZydTpQVFVFfHw8nnjiCaxZswaTJ09GTEwMqqqq8Nprr2Hy5Mk4ePAgXC6XZ2ok8qBOhZQQAlOnTsXq1auxdetWpKSktFs/aNAgaLVa5OXluZeVlpairKwMFosFAGCxWFBcXIyqqip3my1btsBoNCLVA+ftiWTjyQEKrVd5PVZRFKiqipSUFMyZMwdz585F//790dbWhlWrVuH3v/89CgsLPVYnkad0KqRycnLw17/+FR9//DG6desG648/LGxqagIAmEwmTJo0CdOmTcO2bdtQWFiIxx9/HBaLBcOHDwcAjBw5EqmpqRg/fjz279+PzZs34+WXX0ZOTg70nvq2SXSNajlzpsvbCA8Px7hx47B06VLcc889AICvvvoKkydPxvbt29HW1sbTfySNToXU4sWLYbPZcNddd+G6665zPz755BN3m7feegsPPPAAsrOzcccdd8BsNmPVqlXu9RqNBuvXr4dGo4HFYsG4ceMwYcIEzJ4923PvikgyuthYKB6Yrqj+4MEub0NRFGi1WgwYMAAfffQRxo0bh4iICBQVFSEnJwd5eXk89UfS6NLvpPyFv5OiQFNXUoKjc+ag7cefbVytEJMJ6X/5i8dOIQohYLVa8eabb2LRokVwOBxIT0/Hu+++iyFDhnhkH0SX45PfSRFRx2hNJo8cSXmaoigwm82YNWsWpk6dCr1ej/379+PZZ5/lYAqSAkOKyAdCTCYoGo2/y7gsRVFgMBgwY8YMTJo0CeHh4dizZw9mzpyJ8vJyXp8iv2JIEfmAJizMMyElBFwtLV3fziUURYHJZMJ//Md/4P777wcAbN68GQsWLEBra6vH90fUUQwpIh/oykzoFxMuF1prajyyrUspioKkpCS89tprGDhwIBwOB959912sWLGCUyiR3zCkiAKIcDq9FlLA+aDq0aMHcnNzkZycjKamJsydOxf79+/naT/yC4YUka94YESey+GAbfduDxRzZYqi4Be/+AWefvppRERE4Ntvv8Wf//xn1NbWMqjI5xhSRD6SMGZM1zficqGlurrr2/kn9Ho9nnzySYwcORJOpxOffPIJ/vGPfzCkyOcYUkQ+orvC3JQyUhQFcXFxmD59OmJjY2Gz2TB//nycO3fO36VRkGFIEfmI9sf7rnWZED47ohk0aBAmTJgARVHwzTffYNWqVTyaIp9iSBH5SEi3bh7Zjqu5Ga4f58v0No1GgyeffBLp6elwOBx4//33cezYMZ/smwhgSBEFHGdT08/e/NCTLoz2y87ORmhoKEpKSvD555+jra3NJ/snYkgRBRhnQwPa6ut9tj+dTofRo0ejV69eqKurw4YNG1BdXc3TfuQTDCkiH1E0GoRERXV5O03Hj6Px++89UFHHKIqCtLQ03H///VBVFV9++SX279/vs/1TcGNIEfmIJiwMpsGDPbMxHx/FqKqK3/72tzAYDKivr8eaNWt4JEU+wZAi8hVVhdYDR1IX+DokUlNT3Tcv3bx5M86ePevT/VNwYkgR+Yii0XgspJzNzT4/mtLr9Xj44Yeh1WphtVqxdetWn+6fghNDioKeEAJ2ux1VVVXe3ZGiQBMW5pFNtdXWQvh40ldFUTBgwAB0794dra2tyM/P5yg/8jqGFAU9IQS2bNmC9evXe3W2b0/dTRcAHJWVED6+hYaiKOjevTvS09Phcrlw8OBBWK1Wn9ZAwYchRUGvpaUFf/3rX7FhwwbUeHGGcU8699VXPh2GfkF0dDQGDBgArVaL8vJyfPfddz6vgYKLfPezJvKxU6dOoaCgABqNBmVlZYiNjfXoUc/FdPHx0MXFoeXMGa9s39tCQkLQt29fGAwGVFdX48SJExBCeK2/iHgkRUFNCIFNmzahpqYGlZWV2LBhg1dHzWljYqCNjfXa9r1NURT069cPJpMJTU1N+P7779HihTsFE13AkKKgZrfbsWnTJjidTgghsGLFCtTV1Xltf5rwcI8NnhBtbX75rVKPHj0QHR0NIQR++OEHNDY2+rwGCh4MKQpaQgiUlJSgtLTUvez48ePYvn271z78NaGhHgupVj/dNsNgMOD6668HAFRUVMDhcPilDgoODCkKWm1tbfj6669x6tQp97Kmpib8/e9/99opLEWng6LTeWRbLd4eMn8FISEh6NmzJxRFQUVFBZqbm/1SBwUHhhQFrZqaGnz99dftPmSFENi7dy8OHTrklX12ZIBBRUMD1peXY9nRo/jHqVNouMJQc/u+fZ4ur0MURUFycjIAoLq6Gq0+HgpPwYWj+ygoCSFw8uRJ7N279yfrjh8/jt27d2PAgAHQaDQ+relYfT3+sG8fjtfXo9nphFGrRVpUFP44ZAi0avvvlPVeCtKOiI+Ph6IoaGho4A96yat4JEVBSQiBvLy8y/4YtampCRs3bsS5c+e8cm0qxGgE1J/+0ztaX4+ntm9Hic2GJqcTAoCttRXbq6rw7K5dOCvRabWYmBgAvp8/kIIPQ4qCksPhwKeffnrFD9kdO3bghx9+8Mq+Iy2Wyw6eePvQIdiucOpsd3U1tlx07czfIiIi/F0CBQmGFAWlQ4cOua876X4cyKAoCmJiYqCqKs6dO4f169d7Zd/ayEgoPjyN6A2+PA1KwY0hRUHH5XJh9erVaGlpwa233ornn38eGo0GiqJg1qxZGDt2LIxGI1auXOmVaZK0kZFAgH/Ie3OOQ6KLMaQo6FRWVmLnzp249957sWjRIowZMwbqj9eIYmJi8Mc//hEvvfQShBDYtm2bx6+7qGFhiBs16ifLs5KSoL3C6L+eBgPSo6M9WkdX8Ae85Csc3UdB59tvv8Xtt9+OSZMmISkpCaWlpVBVFa2trWhoaEBMTAxycnJw880348CBA2hqakJ4eLjH9q8oChIefBAOqxU127a5l2cmJgIAXt2/Hy1OJ1wANIqCSJ0ObwwZgh4GQ7vtJGRne6ymzrLZbH7bNwUXhhQFndTUVNx6663o1q0bFEVBREQEwsLC4HA4UFVVBSEEwsLCkJmZ6b4TraepYWGIslhg27sXzh+nYVIUBZmJiegeHo71J0/ibHMzehoMeCQlBTF6fbvX66+7DlEWi98mdq2srIQQAqGhoe6jUCJvYEhR0ImLi2v33GAwoFu3bqitrUVFRYV7uaqqiPLg7d4vpigKjLfeirj77kPl//2f+waGiqIgLSoKaT+z35CoKCSOH39+KLufXOin6OhoaLVav9VB1z5+BaKgFxYW5g6uo0eP+uy3P6pej4Rf/QrR99wDJaRj3xc1BgOue+QRRA4d6rcRgk6n032LDrPZDP0lR3lEnsSQoqCnqip69+4N4Pz1Kl9O86MJD0f3J55AQnY2dPHxV2ynaDQI69kTSU89hbj77oPqofn/rkZDQwPKysoAAMnJyQjz0IS5RJfD030U9FRVRd++fQGcn8/v1KlT6NWrl0/2rSgKQiIicN1vfgNjejrO7diB+kOH4LBa4XI4oDEYENq9O0yDB8M0eDDCkpP9foNBq9WKs2fPAjh/247Q0FC/1kPXNoYUBT1VVZGWlgatVouWlhbs27fPZyHlrkGvhyEtDRE33ghnU9P5e0W5XFA0GqhaLdTwcKgdPCXoTUIIfPvtt7DZbNDpdOjTpw9DiryKp/so6CmKgh49esBsNqO1tRW7du3yy5x0iqJA1euhjYyELjYW+vh46GJiEGI0ShFQwPkfQh8+fBh2ux2xsbHo1asXR/eRV/Gvi4KeoiiIj4/HDTfcAKfTieLiYtjtdn+XJaW6ujqUlJSgpaUFZrPZ50ecFHwYUkQ4f+uJAQMGQFVVHDt2DCUlJZzh+xJCCJw6dQpFRUUAgH79+rnvK0XkLQwpIgChoaEYPHgwjEYjTp48iaKiIrhcLn+XJRUhBEpKSvD9999Dq9Xirrvu4vBz8jqGFBHOn/IbOnQo4uPj0dTUhO3bt6Ouro5HUxdxOp3YsmULGhsbYTKZcPfdd/u7JAoCDCmiH6WkpOCOO+4AAOTl5bl/C0TnnTp1CuvWrQMA3HPPPTzVRz7BkCL6kUajwaOPPgqdTofKykqsXbvW3yVJQwiBVatW4cyZMwgNDUV2djbvKUU+wZAiusjAgQMxcOBAAMCyZctQXV3t54r8TwiB06dPY82aNXA6nejfv7+7j4i8jSFFdBGDwYAnnngCer0ex44dw0cffYS2tjZ/l+VXTqcT69atw4EDBxAaGooHH3wQ119/vd9nvqDgwJAiuohGo8Evf/lL3HLLLXA4HFi1ahV++OGHoB1AIYSA1WrFypUrYbfbkZKSglGjRkHnx7kDKbgwpIgukZKSgjFjxiA0NBRFRUX49NNP0dLS4u+y/EIIgU2bNmHHjh3QaDR44IEHcPPNN/u7LAoiDCmiS2i1WowfPx5paWloamrCkiVLcPjw4aA8mjpx4gRyc3PR3NyMlJQU/O53v+P9o8inGFJEl5GYmIgpU6ZAq9XixIkTmD9/PhobG/1dlk81NDTg7bffxsmTJ6HX6/Fv//ZvvBZFPseQIroMRVEwZswYZGZmQlEUrFu3DitWrAiaQRROpxMbN27EypUr4XK5cOeddyI7O5uTyZLP8S+O6AqioqIwffp09O3bF7W1tXjjjTewY8eOa366JCEEDh06hLlz58JqtSIpKQnPP/88kiW4lxUFH4YU0RUoioJBgwbhiSeeQHh4OI4cOYJ58+a5b/h3rbLb7Zg7dy72798PVVXxr//6r7jjjjsYUOQXDCminxEWFoYnn3wSWVlZAIDPP/8cs2fPRmNj4zU3kEIIgfr6erzxxhv49NNPoSgKMjMz8cwzz/DGhuQ3DCmif8JoNGLu3LmwWCxwuVxYunQp3njjjWtqAlohBBobG7Fw4UIsXLgQDocDgwcPxpw5cxAVFeXv8iiIMaSI/glFUZCUlISZM2eib9++aGxsxKJFi/CXv/wFbW1tAR9UQgi4XC4sW7YMb7/9Nmpra3H99dfjpZdeQmpqKk/zkV8xpIg6QKPR4O6778bcuXORmJiIqqoqzJo1C//7v/+LpqamgA0qIQSampqwdOlSvPzyy6iqqkJ0dDRyc3MxatQohEhy23oKXgwpog4KCQlBVlYW3njjDSQlJcFms+HFF1/Em2++ierq6oAMqrq6OsyfPx8vvfQSzpw5g8TERMyePRsPP/wwA4qkwL9Cok4aM2YMmpubMWvWLJSXl+PNN99ERUUFZs+ejdjYWOlPj10I03PnzmHu3Ll47733YLPZEB0djRdffBHjxo1jQJE0eCRF1AmKokCv1+Oxxx7DO++8gxtuuAF2ux3vv/8+xo0bh507d6KlpUXqoyqn04m9e/di0qRJeOedd2C329G9e3csWLAATz75JAwGg/RBS8GDIUXUSYqiQKvVIisrC3/6059w2223AQD+8Y9/YOLEiVi4cKGUv6USQqCmpgYffvghJk6ciLVr16K1tRVDhw7FokWL8PDDD0Or1TKgSCqKkPkr3xXY7XaYTCbYbDYYjUZ/l0NBzOl04vDhw3jttdewcuVKtLa2olu3bhg1ahR+//vfY/DgwX7/4L8wem///v1YsGABNmzYgJqaGmg0GowaNQr/9V//hZtvvpmn+MinOvo53qkjqcWLFyM9PR1GoxFGoxEWiwWbNm1yr29ubkZOTg5iYmJgMBiQnZ2NysrKdtsoKytDVlYWwsPDER8fj+nTpwfNfGh07dFoNEhLS8OSJUvw6quvolevXmhsbMTKlSvx0EMPYfr06SguLkZDQ4PPTwEKIdDc3IySkhLMnj0bWVlZ+Nvf/oZz586hR48emDlzJpYuXYr09HQGFEmrU0dS69atg0ajQZ8+fSCEwIcffoh58+Zh3759uPnmmzFlyhRs2LABS5cuhclkwtSpU6GqKrZv3w7g/LfOW265BWazGfPmzcPp06cxYcIEPPXUU5gzZ06Hi+aRFMmopaUFO3fuxJ///Gd8+umnaGhogKIouOGGG5CVlYUHHngAQ4cOhcFg8HotjY2NKC4uxrp167B27VocOXIETqcT4eHhyMjIwNSpU3H77bdDr9d7vRaiy+nw57jooqioKPHee++J2tpaodVqxYoVK9zrSkpKBABRUFAghBBi48aNQlVVYbVa3W0WL14sjEajcDgcHd6nzWYTAITNZutq+UQe5XK5RFVVlVi6dKkYMmSICA0NFaqqCq1WK3r06CGys7PF0qVLRUVFhXA4HMLpdAqXy9XlfTqdTuFwOITVahXLly8Xjz76qLjxxhuFTqcTqqoKnU4nBg4cKP70pz+JioqKLu+TqKs6+jl+1cf4TqcTK1asQENDAywWCwoLC9Ha2oqMjAx3m379+iE5ORkFBQUYPnw4CgoK0L9/fyQkJLjbZGZmYsqUKTh06BBuvfXWy+7L4XDA4XC0S2AiGSmKgtjYWIwfPx73338/VqxYgY8++gilpaUoLy/HyZMnsXbtWpjNZtx+++248847kZaWhvj4eJhMJhiNRuj1+p+9hiWEQGtrK+x2O86dO4fq6mocPnwYW7duRUFBASoqKuB0OiGEgNFoxE033YTf/OY3GD9+PGJiYqAoCgdHUMDodEgVFxfDYrGgubkZBoMBq1evRmpqKoqKiqDT6RAZGdmufUJCAqxWKwDAarW2C6gL6y+su5Lc3Fy88sornS2VyC8uhEBcXBwmT56M+++/H1988QU+++wzfPnll6iqqkJFRQU++eQT/P3vf0dsbCx69OiBpKQkdO/eHXFxcYiOjobRaERoaChCQkLgcrnQ3NyMuro6VFdXo7q6GhUVFThx4gTKyspQVVXlvuZ1Yd8WiwWjR4/G3XffjeTkZN4LigJSp0Oqb9++KCoqgs1mw8qVKzFx4kTk5+d7oza3GTNmYNq0ae7ndrsdSUlJXt0nkSeoqoqePXtiwoQJGD16NI4ePYqvvvoKa9euxcGDB9HY2IizZ8/izJkz2Lt3L4DzM1vodDrodDpoNBooigIhBJxOJ1pbW+FwONoNNlJVFVqtFhEREejduzcefPBB3HPPPejVqxdiYmIYThTQOh1SOp0OvXv3BgAMGjQIe/bswfz58/HII4+gpaUFtbW17Y6mKisrYTabAQBmsxm7d+9ut70Lo/8utLkcvV7PC7wU0FRVRXR0NKKjozFo0CDk5OTg6NGj2LNnD3bt2oWjR4+iuroa586dQ0NDAxobG9HY2OiewFZVVWg0Guh0OkRHRyMiIgImkwnx8fFITk7G0KFDMXz4cPTq1avd6UKe1qNA1+Vxpy6XCw6HA4MGDYJWq0VeXh6ys7MBAKWlpSgrK4PFYgEAWCwW/Pd//zeqqqoQHx8PANiyZQuMRiNSU1O7WgqR1C4ODr1ej5tuugk33XQTxo0bB5vNhjNnzqCyshLnzp1DXV0dmpqa0NraCpfLBVVVodPpEBoaCqPRiMjISMTFxbmvZWk0Gj+/OyLv6FRIzZgxA/fddx+Sk5NRV1eHjz/+GF988QU2b94Mk8mESZMmYdq0ae7z6c888wwsFguGDx8OABg5ciRSU1Mxfvx4vP7667BarXj55ZeRk5PDIyUKWqqqIioqClFRUbjxxhv9XQ6RVDoVUlVVVZgwYQJOnz4Nk8mE9PR0bN68Gffeey8A4K233oKqqsjOzobD4UBmZiYWLVrkfr1Go8H69esxZcoUWCwWREREYOLEiZg9e7Zn3xUREV0TOC0SERH5nFemRSIiIvIlhhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNLqUkjNnTsXiqLgueeecy9rbm5GTk4OYmJiYDAYkJ2djcrKynavKysrQ1ZWFsLDwxEfH4/p06ejra2tK6UQEdE16KpDas+ePXj33XeRnp7ebvnzzz+PdevWYcWKFcjPz8epU6fw61//2r3e6XQiKysLLS0t2LFjBz788EMsXboUs2bNuvp3QURE1yZxFerq6kSfPn3Eli1bxJ133imeffZZIYQQtbW1QqvVihUrVrjblpSUCACioKBACCHExo0bhaqqwmq1utssXrxYGI1G4XA4OrR/m80mAAibzXY15RMRkZ919HP8qo6kcnJykJWVhYyMjHbLCwsL0dra2m55v379kJycjIKCAgBAQUEB+vfvj4SEBHebzMxM2O12HDp06LL7czgcsNvt7R5ERHTtC+nsC5YvX45vvvkGe/bs+ck6q9UKnU6HyMjIdssTEhJgtVrdbS4OqAvrL6y7nNzcXLzyyiudLZWIiAJcp46kysvL8eyzz+Jvf/sbQkNDvVXTT8yYMQM2m839KC8v99m+iYjIfzoVUoWFhaiqqsLAgQMREhKCkJAQ5OfnY8GCBQgJCUFCQgJaWlpQW1vb7nWVlZUwm80AALPZ/JPRfheeX2hzKb1eD6PR2O5BRETXvk6F1IgRI1BcXIyioiL3Y/DgwRg7dqz7v7VaLfLy8tyvKS0tRVlZGSwWCwDAYrGguLgYVVVV7jZbtmyB0WhEamqqh94WERFdCzp1Tapbt25IS0trtywiIgIxMTHu5ZMmTcK0adMQHR0No9GIZ555BhaLBcOHDwcAjBw5EqmpqRg/fjxef/11WK1WvPzyy8jJyYFer/fQ2yIiomtBpwdO/DNvvfUWVFVFdnY2HA4HMjMzsWjRIvd6jUaD9evXY8qUKbBYLIiIiMDEiRMxe/ZsT5dCREQBThFCCH8X0Vl2ux0mkwk2m43Xp4iIAlBHP8c5dx8REUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSSvE3wVcDSEEAMBut/u5EiIiuhoXPr8vfJ5fSUCG1NmzZwEASUlJfq6EiIi6oq6uDiaT6YrrAzKkoqOjAQBlZWU/++aCnd1uR1JSEsrLy2E0Gv1djrTYTx3DfuoY9lPHCCFQV1eHxMTEn20XkCGlqucvpZlMJv4RdIDRaGQ/dQD7qWPYTx3DfvrnOnKQwYETREQkLYYUERFJKyBDSq/X4w9/+AP0er2/S5Ea+6lj2E8dw37qGPaTZynin43/IyIi8pOAPJIiIqLgwJAiIiJpMaSIiEhaDCkiIpJWQIbUwoUL0bNnT4SGhmLYsGHYvXu3v0vyqS+//BKjR49GYmIiFEXBmjVr2q0XQmDWrFm47rrrEBYWhoyMDHz33Xft2tTU1GDs2LEwGo2IjIzEpEmTUF9f78N34V25ubkYMmQIunXrhvj4eDz00EMoLS1t16a5uRk5OTmIiYmBwWBAdnY2Kisr27UpKytDVlYWwsPDER8fj+nTp6Otrc2Xb8WrFi9ejPT0dPcPTy0WCzZt2uRezz66vLlz50JRFDz33HPuZewrLxEBZvny5UKn04n3339fHDp0SDz11FMiMjJSVFZW+rs0n9m4caN46aWXxKpVqwQAsXr16nbr586dK0wmk1izZo3Yv3+/GDNmjEhJSRFNTU3uNqNGjRIDBgwQO3fuFF999ZXo3bu3eOyxx3z8TrwnMzNTfPDBB+LgwYOiqKhI3H///SI5OVnU19e720yePFkkJSWJvLw8sXfvXjF8+HDxi1/8wr2+ra1NpKWliYyMDLFv3z6xceNGERsbK2bMmOGPt+QVa9euFRs2bBDffvutKC0tFS+++KLQarXi4MGDQgj20eXs3r1b9OzZU6Snp4tnn33WvZx95R0BF1JDhw4VOTk57udOp1MkJiaK3NxcP1blP5eGlMvlEmazWcybN8+9rLa2Vuj1erFs2TIhhBCHDx8WAMSePXvcbTZt2iQURREVFRU+q92XqqqqBACRn58vhDjfJ1qtVqxYscLdpqSkRAAQBQUFQojzXwZUVRVWq9XdZvHixcJoNAqHw+HbN+BDUVFR4r333mMfXUZdXZ3o06eP2LJli7jzzjvdIcW+8p6AOt3X0tKCwsJCZGRkuJepqoqMjAwUFBT4sTJ5HDt2DFartV0fmUwmDBs2zN1HBQUFiIyMxODBg91tMjIyoKoqdu3a5fOafcFmswH4/5MTFxYWorW1tV0/9evXD8nJye36qX///khISHC3yczMhN1ux6FDh3xYvW84nU4sX74cDQ0NsFgs7KPLyMnJQVZWVrs+Afj35E0BNcFsdXU1nE5nu//JAJCQkIAjR474qSq5WK1WALhsH11YZ7VaER8f3259SEgIoqOj3W2uJS6XC8899xxuu+02pKWlATjfBzqdDpGRke3aXtpPl+vHC+uuFcXFxbBYLGhubobBYMDq1auRmpqKoqIi9tFFli9fjm+++QZ79uz5yTr+PXlPQIUU0dXIycnBwYMH8fXXX/u7FCn17dsXRUVFsNlsWLlyJSZOnIj8/Hx/lyWV8vJyPPvss9iyZQtCQ0P9XU5QCajTfbGxsdBoND8ZMVNZWQmz2eynquRyoR9+ro/MZjOqqqrarW9ra0NNTc01149Tp07F+vXrsW3bNnTv3t293Gw2o6WlBbW1te3aX9pPl+vHC+uuFTqdDr1798agQYOQm5uLAQMGYP78+eyjixQWFqKqqgoDBw5ESEgIQkJCkJ+fjwULFiAkJAQJCQnsKy8JqJDS6XQYNGgQ8vLy3MtcLhfy8vJgsVj8WJk8UlJSYDab2/WR3W7Hrl273H1ksVhQW1uLwsJCd5utW7fC5XJh2LBhPq/ZG4QQmDp1KlavXo2tW7ciJSWl3fpBgwZBq9W266fS0lKUlZW166fi4uJ2gb5lyxYYjUakpqb65o34gcvlgsPhYB9dZMSIESguLkZRUZH7MXjwYIwdO9b93+wrL/H3yI3OWr58udDr9WLp0qXi8OHD4umnnxaRkZHtRsxc6+rq6sS+ffvEvn37BADx5ptvin379okTJ04IIc4PQY+MjBSffvqpOHDggHjwwQcvOwT91ltvFbt27RJff/216NOnzzU1BH3KlCnCZDKJL774Qpw+fdr9aGxsdLeZPHmySE5OFlu3bhV79+4VFotFWCwW9/oLQ4ZHjhwpioqKxGeffSbi4uKuqSHDL7zwgsjPzxfHjh0TBw4cEC+88IJQFEV8/vnnQgj20c+5eHSfEOwrbwm4kBJCiP/5n/8RycnJQqfTiaFDh4qdO3f6uySf2rZtmwDwk8fEiROFEOeHoc+cOVMkJCQIvV4vRowYIUpLS9tt4+zZs+Kxxx4TBoNBGI1G8fjjj4u6ujo/vBvvuFz/ABAffPCBu01TU5P43e9+J6KiokR4eLj41a9+JU6fPt1uO8ePHxf33XefCAsLE7GxseLf//3fRWtrq4/fjfc88cQTokePHkKn04m4uDgxYsQId0AJwT76OZeGFPvKO3irDiIiklZAXZMiIqLgwpAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWgwpIiKS1v8D13+1+RjFh1wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "-120.60388566888659"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test(play=True)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第7章-DQN算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
